{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-09-26T03:22:13.098009Z",
     "start_time": "2025-09-26T03:22:13.087909Z"
    }
   },
   "source": "print('hello xlgeng')",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello xlgeng\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-26T03:23:31.633131Z",
     "start_time": "2025-09-26T03:23:31.338609Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from transformers import AutoConfig\n",
    "\n",
    "# 加载 bert-base-uncased 的配置\n",
    "config = AutoConfig.from_pretrained(\"bert-base-uncased\")\n",
    "\n",
    "# 打印配置的部分内容\n",
    "print(f\"模型隐藏层维度: {config.hidden_size}\")\n",
    "print(f\"注意力头数量: {config.num_attention_heads}\")\n",
    "print(f\"隐藏层: {config.num_hidden_layers}\")\n",
    "print(config)\n"
   ],
   "id": "15a304f23f6b0022",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "模型隐藏层维度: 768\n",
      "注意力头数量: 12\n",
      "隐藏层: 12\n",
      "BertConfig {\n",
      "  \"_name_or_path\": \"bert-base-uncased\",\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.44.0\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-26T03:23:59.293845Z",
     "start_time": "2025-09-26T03:23:57.500061Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from transformers import AutoTokenizer, AutoModel\n",
    "# 修改配置，创建一个只有6层的BERT模型\n",
    "config.num_hidden_layers = 6\n",
    "# 之后可以用这个新的 config 来初始化模型\n",
    "model = AutoModel.from_config(config)"
   ],
   "id": "cb137713619ea377",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-26T03:24:02.473788Z",
     "start_time": "2025-09-26T03:24:02.470239Z"
    }
   },
   "cell_type": "code",
   "source": "print(model)",
   "id": "2f3fdcd2e7a11f54",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertModel(\n",
      "  (embeddings): BertEmbeddings(\n",
      "    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
      "    (position_embeddings): Embedding(512, 768)\n",
      "    (token_type_embeddings): Embedding(2, 768)\n",
      "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "    (dropout): Dropout(p=0.1, inplace=False)\n",
      "  )\n",
      "  (encoder): BertEncoder(\n",
      "    (layer): ModuleList(\n",
      "      (0-5): 6 x BertLayer(\n",
      "        (attention): BertAttention(\n",
      "          (self): BertSelfAttention(\n",
      "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "          (output): BertSelfOutput(\n",
      "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "            (dropout): Dropout(p=0.1, inplace=False)\n",
      "          )\n",
      "        )\n",
      "        (intermediate): BertIntermediate(\n",
      "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (intermediate_act_fn): GELUActivation()\n",
      "        )\n",
      "        (output): BertOutput(\n",
      "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (pooler): BertPooler(\n",
      "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "    (activation): Tanh()\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-26T03:25:06.476111Z",
     "start_time": "2025-09-26T03:25:06.089330Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "print(tokenizer.vocab_size)\n",
    "print(tokenizer)"
   ],
   "id": "b72660a89441bd81",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30522\n",
      "BertTokenizerFast(name_or_path='bert-base-uncased', vocab_size=30522, model_max_length=512, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True),  added_tokens_decoder={\n",
      "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "}\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-26T04:12:36.959054Z",
     "start_time": "2025-09-26T04:12:36.952681Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from transformers import (AutoConfig, AutoModel, HfArgumentParser, Trainer,\n",
    "                          TrainerCallback, TrainingArguments)\n",
    "from dataclasses import dataclass, field\n",
    "config_path = \"/Users/xuelonggeng/Documents/code/west_xlgeng/examples/aishell/asr/conf/qwen2-7b_firered.json\""
   ],
   "id": "5ea892bfa2d28ff1",
   "outputs": [],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-26T06:37:35.426684Z",
     "start_time": "2025-09-26T06:37:35.392586Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import west\n",
    "\n",
    "config_new = AutoConfig.from_pretrained(config_path)\n",
    "print(config_new)"
   ],
   "id": "c45802c49ff663b6",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TouchASUConfig {\n",
      "  \"_name_or_path\": \"/Users/xuelonggeng/Documents/code/west_xlgeng/examples/aishell/asr/conf/qwen2-7b_firered.json\",\n",
      "  \"encoder_ds_rate\": 4,\n",
      "  \"encoder_projector_ds_rate\": 2,\n",
      "  \"hidden_size\": 0,\n",
      "  \"llm_model_name_or_path\": \"Qwen/Qwen2-7B-Instruct\",\n",
      "  \"lora_config\": null,\n",
      "  \"max_speech_frames\": 2000,\n",
      "  \"min_speech_frames\": 20,\n",
      "  \"model_type\": \"touch_asu\",\n",
      "  \"projector_hidden_size\": 2048,\n",
      "  \"transformers_version\": \"4.44.0\",\n",
      "  \"wenet_model_name_or_path\": \"firered\"\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-26T06:47:23.755742Z",
     "start_time": "2025-09-26T06:47:18.330091Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import west\n",
    "from transformers import (AutoConfig, AutoModel, HfArgumentParser, Trainer,\n",
    "                          TrainerCallback, TrainingArguments)\n",
    "osum_config_path = (\"/Users/xuelonggeng/Documents/code/west_xlgeng/examples/aishell/asr/conf/osum_echat.json\")\n",
    "config_new = AutoConfig.from_pretrained(osum_config_path)\n",
    "print(config_new)"
   ],
   "id": "d731d42921152e25",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/xlgeng/lib/python3.10/site-packages/requests/__init__.py:86: RequestsDependencyWarning: Unable to find acceptable character detection dependency (chardet or charset_normalizer).\n",
      "  warnings.warn(\n",
      "/opt/anaconda3/envs/xlgeng/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "/bin/sh: lscpu: command not found\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Module \"torch_npu\" not found. \"pip install torch_npu\"                 if you are using Ascend NPU, otherwise, ignore it\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/xlgeng/lib/python3.10/site-packages/torch/amp/autocast_mode.py:250: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OSUMEChatConfig {\n",
      "  \"_name_or_path\": \"/Users/xuelonggeng/Documents/code/west_xlgeng/examples/aishell/asr/conf/osum_echat.json\",\n",
      "  \"encoder_ds_rate\": 2,\n",
      "  \"encoder_projector_ds_rate\": 4,\n",
      "  \"hidden_size\": 0,\n",
      "  \"llm_model_name_or_path\": \"Qwen/Qwen2.5-3B-Instruct\",\n",
      "  \"lora_config\": null,\n",
      "  \"max_speech_frames\": 2000,\n",
      "  \"min_speech_frames\": 20,\n",
      "  \"model_type\": \"osum_echat\",\n",
      "  \"no_init_llm\": true,\n",
      "  \"projector_hidden_size\": 2048,\n",
      "  \"transformers_version\": \"4.44.0\",\n",
      "  \"wenet_model_name_or_path\": \"whisper-medium\"\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-26T06:48:33.699749Z",
     "start_time": "2025-09-26T06:48:33.440542Z"
    }
   },
   "cell_type": "code",
   "source": [
    "osum_model = AutoModel.from_config(config_new)\n",
    "print(osum_model)"
   ],
   "id": "27172889c41528a4",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OSUMEChat()\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "1b3016a68eb33d4b"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
